# name: 朱汉青
# ID: 2201210568
# Gitee URL: https://gitee.com/hanqing_zhu/python-final-project

import re
import pandas as pd
import os


def task1(abs_path):

    ma_path = abs_path + r"/data/m-a_edges.csv"
    df = pd.read_csv(ma_path, encoding='utf-8', header=None, sep='\t')
    df = df.iloc[1:]
    df.columns = ["mashup", "api"]

    # Part1 & 2:
    # API in Mashup
    mashup = df.groupby("mashup").count()
    mashup = mashup.sort_values("api", ascending=False)

    # Mashup in API
    api = df.groupby("api").count()
    api = api.sort_values("mashup", ascending=False)

    # count API and Mashup
    api_unique = df["api"].unique()
    mashup_unique = df["mashup"].unique()

    print(mashup)
    print(api)
    print(len(api_unique))
    print(len(mashup_unique))

    # Part3: count API Provider
    api_accessibility_path = abs_path + "/data/raw/accessibility/api_accessibility/"
    providers = {}
    findURL = re.compile(r'"visit_url": "https?://(.*)"')
    for file_name in os.listdir(api_accessibility_path):
        file_path = api_accessibility_path + file_name
        with open(file_path, "r", encoding="utf-8") as file:
            content = file.read()
            visit_url = re.findall(findURL, content)
            for url in visit_url:
                idx = url.find('/')
                length = len(url)
                if idx != -1:
                    url = url[0:idx]
                if providers.get(url) is not None:
                    providers[url] += 1
                else:
                    providers.update({url: 1})
    providers_names = list(providers.keys())
    api_count = list(providers.values())
    providers_df = pd.DataFrame(api_count, providers_names)
    providers_df.columns = ['api_count']
    api_providers_count = providers_df.sort_values('api_count', ascending=False)
    print(api_providers_count)


def task2(abs_path):
    mashup_path = abs_path + r"/data/mashup_nodes_estimator.csv"
    mashup_df = pd.read_csv(mashup_path, encoding='utf-8', header=None, sep='\t')
    mashup_df = mashup_df.iloc[1:]
    mashup_df.columns = ["type", "url", "mashup_name", "sd", "et", "oet", "mashup_category", "oac", "ac"]
    mashup_df.drop(["type", "url", "sd", "et", "oet", "oac", "ac"], axis=1, inplace=True)
    mashup_categories = mashup_df["mashup_category"].unique()

    api_path = abs_path + r"/data/api_nodes_estimator.csv"
    api_df = pd.read_csv(api_path, encoding='utf-8', header=None, sep='\t')
    api_df = api_df.iloc[1:]
    api_df.columns = ["type", "api_url", "api_name", "sd", "et", "oet", "api_category", "oac", "ac"]
    api_df.drop(["type", "api_name", "sd", "et", "oet", "oac", "ac"], axis=1, inplace=True)
    api_categories = api_df["api_category"].unique()

    ma_path = abs_path + r"/data/m-a_edges.csv"
    ma_df = pd.read_csv(ma_path, encoding='utf-8', header=None, sep='\t')
    ma_df = ma_df.iloc[1:]
    ma_df.columns = ["mashup", "api"]

    ma_df = pd.merge(mashup_df, ma_df, left_on="mashup_name", right_on="mashup", how="inner")
    ma_df = pd.merge(ma_df, api_df, left_on="api", right_on="api_url", how="inner")
    ma_df.drop(["mashup", "api"], axis=1, inplace=True)

    mashup_df = ma_df.set_index(["mashup_category", "mashup_name"])
    mashup_df.sort_index(inplace=True)

    api_df = ma_df.set_index(["api_category", "api_url"])
    api_df.sort_index(inplace=True)

    # API MOST related to Mashup
    mashup_related_df = pd.DataFrame(columns=
                              ["category", "related_category", "category_size", "related_category_size", "rate"])

    for category in mashup_categories:
        if category in mashup_df.index:
            df = mashup_df.loc[category]
            df_size = len(df)
            if df_size < 10:
                continue
            result = df["api_category"].value_counts()
            index = result.index[0]
            if index == 'Other':
                index = result.index[1]
            max_num = result[index]
            rate = max_num/df_size
            mashup_related_df.loc[len(mashup_related_df)] = [category, index, df_size, max_num, rate]

    mashup_related_df.set_index("category", inplace=True)
    mashup_related_df.sort_values(by="rate", inplace=True, ascending=False)
    print(mashup_related_df.head(5))

    # Mashup MOST related to API
    api_related_df = pd.DataFrame(columns=
                                     ["category", "related_category", "category_size", "related_category_size", "rate"])

    for category in api_categories:
        if category == "Other":
            continue
        if category in api_df.index:
            df = api_df.loc[category]
            df_size = len(df)
            if df_size < 10:
                continue
            result = df["mashup_category"].value_counts()
            index = result.index[0]
            if index == 'Other':
                index = result.index[1]
            max_num = result[index]
            rate = max_num / df_size
            api_related_df.loc[len(api_related_df)] = [category, index, df_size, max_num, rate]

    api_related_df.set_index("category", inplace=True)
    api_related_df.sort_values(by="rate", inplace=True, ascending=False)
    print(api_related_df.head(5))


def task3(abs_path):

    api_path = abs_path + r"/data/api_nodes_estimator.csv"
    api_df = pd.read_csv(api_path, encoding='utf-8', header=None, sep='\t')
    api_df = api_df.iloc[1:]
    api_df.columns = ["type", "api_url", "api_name", "sd", "et", "oet", "api_category", "oac", "ac"]
    api_df.drop(["type", "sd", "et", "oet", "oac", "ac"], axis=1, inplace=True)
    api_df['style'] = ['Unknown']*len(api_df)
    api_categories = api_df["api_category"].unique()

    ma_path = abs_path + r"/data/m-a_edges.csv"
    ma_df = pd.read_csv(ma_path, encoding='utf-8', header=None, sep='\t')
    ma_df = ma_df.iloc[1:]
    ma_df.columns = ["mashup", "api"]

    api_url = api_df['api_url'].tolist()
    api_list = ma_df['api'].tolist()
    api_url_used = []
    for url in api_url:
        if url in api_list:
            times = ma_df['api'].value_counts()[url]
            api_url_used.append(times)
        else:
            api_url_used.append(0)
    api_df['used'] = api_url_used

    alive_path = abs_path + r"/data/raw/api_mashup/active_apis_data.txt"

    findAPIsData = re.compile(r'"versions"(.*\n.*\n.*\n.*\n.*\n.*\n)')
    findTitle = re.compile(r'"title": "(.*)"')
    findStyle = re.compile(r'"style": "(.*)"')
    with open(alive_path, "r", encoding="utf-8") as file:
        content = file.read()
        datas = re.findall(findAPIsData, content)
        titles = re.findall(findTitle, content)
    styles = []
    for data in datas:
        style = re.findall(findStyle, data)
        if len(style) > 0:
            styles.append(style[0])
    wrong_titles = ['Change Healthcare API MASTER RECORD', 'Ion Channel API MASTER RECORD', 'Ion Channel API MASTER RECORD']
    for wrong_title in wrong_titles:
        titles.remove(wrong_title)
    for index in range(len(titles)):
        title = titles[index]
        style = styles[index]
        idx = api_df[api_df['api_name'] == title].index.tolist()
        if len(idx) > 0:
            idx = idx[0]
            api_df.loc[idx, 'style'] = style

    api_df = api_df.set_index(["api_category", "api_url"])
    api_df.sort_index(inplace=True)

    api_related_df = pd.DataFrame(columns=
                                  ["category", "protocol", "category_size", "cur_protocol_size", "rate"])
    most_used_data = pd.DataFrame(columns=['category', 'api_url', 'api_name', 'style', 'used'])

    for category in api_categories:
        if category == "Other":
            continue
        if category in api_df.index:
            df = api_df.loc[category]
            df_size = len(df)
            if df_size < 10:
                continue
            result = df["style"].value_counts()

            protocol = result.index[0]
            if protocol == 'Unknown':
                protocol = result.index[1]
            max_num = result[protocol]
            rate = max_num / df_size
            api_related_df.loc[len(api_related_df)] = [category, protocol, df_size, max_num, rate]

            df = df.copy()
            df.sort_values(by="used", inplace=True, ascending=False)
            df = df.reset_index()

            most_used_data.loc[len(most_used_data)] = [category, df.loc[0, 'api_url'], df.loc[0, 'api_name'], df.loc[0, 'style'], df.loc[0, 'used']]

    api_related_df.set_index("category", inplace=True)
    api_related_df.sort_values(by="rate", inplace=True, ascending=False)
    print(api_related_df)

    most_used_data.sort_values(by='used', inplace=True, ascending=False)
    print(most_used_data.head(10))


def main():
    abs_path = ""
    task1(abs_path)
    task2(abs_path)
    task3(abs_path)


if __name__ == '__main__':
    main()
